<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Delaying Shard Allocation | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="post_deploy.html" title="Post-Deployment">
<link rel="prev" href="indexing-performance.html" title="Indexing Performance Tips">
<link rel="next" href="_rolling_restarts.html" title="Rolling Restarts">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="administration.html">Administration, Monitoring, and Deployment</a></span>
»
<span class="breadcrumb-link"><a href="post_deploy.html">Post-Deployment</a></span>
»
<span class="breadcrumb-node">Delaying Shard Allocation</span>
</div>
<div class="navheader">
<span class="prev">
<a href="indexing-performance.html">« Indexing Performance Tips</a>
</span>
<span class="next">
<a href="_rolling_restarts.html">Rolling Restarts »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="_delaying_shard_allocation"></a>Delaying Shard Allocation<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/520_Post_Deployment/35_delayed_shard_allocation.asciidoc">edit</a>
</h2>
</div></div></div>
<p>As discussed way back in <a class="xref" href="_scale_horizontally.html" title="Scale Horizontally">Scale Horizontally</a>, Elasticsearch will automatically
balance shards between your available nodes, both when new nodes are added and
when existing nodes leave.</p>
<p>Theoretically, this is the best thing to do.  We want to recover missing primaries
by promoting replicas as soon as possible.  We also want to make sure resources
are balanced evenly across the cluster to prevent hotspots.</p>
<p>In practice, however, immediately re-balancing can cause more problems than it solves.
For example, consider this situation:</p>
<div class="olist orderedlist">
<ol class="orderedlist">
<li class="listitem">
Node 19 loses connectivity to your network (someone tripped on the power cable)
</li>
<li class="listitem">
Immediately, the master notices the node departure.  It determines
what primary shards were on Node 19 and promotes the corresponding replicas around
the cluster
</li>
<li class="listitem">
After replicas have been promoted to primary, the master begins issuing recovery
commands to rebuild the now-missing replicas.  Nodes around the cluster fire up
their NICs and start pumping shard data to each other in an attempt to get back
to green health status
</li>
<li class="listitem">
This process will likely trigger a small cascade of shard movement, since the
cluster is now unbalanced.  Unrelated shards will be moved between hosts to accomplish
better balancing
</li>
</ol>
</div>
<p>Meanwhile, the hapless admin who kicked out the power cable plugs it back in.
Node 19 reboots and rejoins the cluster.  Unfortunately, the node is informed that
its existing data is now useless; the data being re-allocated elsewhere.
So Node 19 deletes its local data and begins recovering a different
set of shards from the cluster (which then causes a new minor re-balancing dance).</p>
<p>If this all sounds needless and expensive, you’re right.  It is, but <em>only when
you know the node will be back soon</em>.  If Node 19 was truly gone, the above procedure
is exactly what we want to happen.</p>
<p>To help address these transient outages, Elasticsearch has the ability to delay
shard allocation.  This gives your cluster time to see if nodes will rejoin before
starting the re-balancing dance.</p>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_changing_the_default_delay"></a>Changing the default delay<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/520_Post_Deployment/35_delayed_shard_allocation.asciidoc">edit</a>
</h3>
</div></div></div>
<p>By default, the cluster will wait one minute to see if the node will rejoin.  If
the node rejoins before the timer expires, the rejoining node will use its existing
shards and no shard allocation occurs.</p>
<p>This default time can be changed either globally, or on a per-index basis, by
configuring the <code class="literal">delayed_timeout</code> setting:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT /_all/_settings <a id="CO309-1"></a><i class="conum" data-value="1"></i>
{
  "settings": {
    "index.unassigned.node_left.delayed_timeout": "5m" <a id="CO309-2"></a><i class="conum" data-value="2"></i>
  }
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO309-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>By using the <code class="literal">_all</code> index name, we can apply this setting to all indices
in the cluster</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO309-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The default time is changed to 5 minutes</p>
</td>
</tr>
</table>
</div>
<p>The setting is dynamic and can be changed at runtime.  If you would like shards to
allocate immediately instead of waiting, you can set <code class="literal">delayed_timeout: 0</code>.</p>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Delayed allocation won’t prevent replicas from being promoted to primaries.
The cluster will still perform promotions as necessary to get the cluster back to
<code class="literal">yellow</code> status.  The allocation of the now-missing replicas will be the only process
that is delayed</p>
</div>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_auto_cancellation_of_shard_relocation"></a>Auto-cancellation of shard relocation<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/520_Post_Deployment/35_delayed_shard_allocation.asciidoc">edit</a>
</h3>
</div></div></div>
<p>What happens if the node comes back <em>after</em> the timeout expires, but before
the cluster has finished moving shards around?  In this case, Elasticsearch will
check to see if the on-disk data matches the current "live" data in the primary shard.
If the two shards are identical — meaning there have been no new documents, updates
or deletes — the master will cancel the on-going rebalancing and restore the
on-disk data.</p>
<p>This is done since recovery of on-disk data will always be faster
than transferring over the network, and since we can guarantee the shards are identical,
the process is a win-win.</p>
<p>If the shards have diverged (e.g. new documents have been indexed since the node
went down), the recovery process will continue as normal.  The rejoining node
will delete it’s local, out-dated shards and obtain a new set.</p>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="indexing-performance.html">« Indexing Performance Tips</a>
</span>
<span class="next">
<a href="_rolling_restarts.html">Rolling Restarts »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
